﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;

namespace Crawler_Plus
{
    class Crawler
    {
        public Hashtable urls = new Hashtable();
        public int count = 0;
        public event Action<Crawler> Crawler_Stop;
        public event Action<Crawler, int, String, String> Crawler_Download;
        public int DowloadNumber=0;
        

        public  void Crawl()   //主线程
        {
            //Console.WriteLine("开始爬行了.... ");
            List<Task> tasks = new List<Task>();
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url])
                        continue;  //已经下载过就不再下载，即value为true
                    current = url;

                }

                if (current == null || count > 10) break;
                Console.WriteLine("爬行" + current + "页面!");
                int index = tasks.Count;
                Task task = Task.Run(() => Dispose(current, index));
                tasks.Add(task);
                string html = DownLoad(current); // 下载
                urls[current] = true;
                count++;
                Parse(html,current);//解析,并加入新的链接
                Console.WriteLine("爬行结束");
            }
            Task.WaitAll(tasks.ToArray());
            Crawler_Stop(this);
        }
        public void Dispose(String url, int index)
        {
            try
            {
                string html = DownLoad(url);
                urls[url] = true;
                count++;
                Parse(html, url);
                Crawler_Download(this, index, url, "success");
            }
            catch (Exception e)
            {
                Crawler_Download(this, index, url, "false");
            }
        }
        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = @"D:\Result\" + count.ToString();
                string fileName1 = @"D:\zyl\" + count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                File.WriteAllText(fileName1, url, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }

        private void Parse(string html, string url)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1)
                            .Trim('"', '\"', '#', '>');
                if (strRef.Length == 0)
                {
                    continue;
                }
                try
                {
                    Uri uri = new Uri(strRef);
                    if (!uri.IsAbsoluteUri)
                    {   //相对地址转化为绝对地址
                        Uri uri1 = new Uri(new Uri(url), strRef);
                        strRef = uri1.ToString();
                    }
                }
                catch (Exception e)
                {
                    continue;
                }

                if (urls[strRef] == null && Choose(strRef))
                    urls[strRef] = false;
            }
        }
        private bool Choose(String url)
        {  //判断是否为html/htm/aspx/jsp网页
            Regex regex = new Regex("(.html|.htm|.aspx|.jsp)");
            if (regex.Matches(url).Count == 0)
                return true;
            else
                return false;

        }
    }
}
